package com.hsl.housaileibot001.crawler;

import com.alibaba.excel.EasyExcel;
import com.hsl.housaileibot001.model.InterviewQuestion;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;

/**
 * 面试题目爬虫
 * 从 mianshiya.com 爬取面试题目数据并保存到Excel
 * @author liul
 * @date 2025/01/27
 */
@Slf4j
public class InterviewQuestionCrawler {
    
    private static final String BASE_URL = "https://www.mianshiya.com/search/all?pageSize=20&current=";
    private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36";
    private static final int TIMEOUT = 10000; // 10秒超时
    private static final int DELAY_MS = 1000; // 2秒延迟，避免被反爬虫检测
    
    /**
     * 爬取指定页面的面试题目数据
     * @param page 页码（从1开始）
     * @return 面试题目列表
     */
    public List<InterviewQuestion> crawlPage(int page) {
        List<InterviewQuestion> questions = new ArrayList<>();
        String url = BASE_URL + page;
        
        try {
            log.info("开始爬取第{}页数据，URL: {}", page, url);
            
            Document document = Jsoup.connect(url)
                    .userAgent(USER_AGENT)
                    .timeout(TIMEOUT)
                    .get();
            
            // 解析表格行
            Elements rows = document.select("tr.ant-table-row");
            
            for (Element row : rows) {
                try {
                    InterviewQuestion question = parseQuestionRow(row);
                    if (question != null) {
                        questions.add(question);
                    }
                } catch (Exception e) {
                    log.warn("解析行数据失败: {}", e.getMessage());
                }
            }
            
            log.info("第{}页爬取完成，获取到{}条数据", page, questions.size());
            
        } catch (IOException e) {
            log.error("爬取第{}页失败: {}", page, e.getMessage());
        }
        
        return questions;
    }
    
    /**
     * 解析表格行，提取题目信息
     * @param row 表格行元素
     * @return 面试题目对象
     */
    private InterviewQuestion parseQuestionRow(Element row) {
        Elements cells = row.select("td.ant-table-cell");
        if (cells.size() < 4) {
            return null;
        }
        
        // 提取题目和链接（第2列）
        Element titleCell = cells.get(1);
        Element titleLink = titleCell.selectFirst("a");
        if (titleLink == null) {
            return null;
        }
        
        String title = titleLink.text().trim();
        String link = "https://www.mianshiya.com" + titleLink.attr("href");
        
        // 提取难度（第3列）
        Element difficultyCell = cells.get(2);
        Element difficultySpan = difficultyCell.selectFirst("span");
        String difficulty = difficultySpan != null ? difficultySpan.text().trim() : "";
        
        // 提取标签（第4列）
        Element tagsCell = cells.get(3);
        Elements tagElements = tagsCell.select("span.ant-tag");
        List<String> tagList = new ArrayList<>();
        for (Element tag : tagElements) {
            tagList.add(tag.text().trim());
        }
        String tags = String.join(", ", tagList);
        
        // 创建面试题目对象并设置来源
        InterviewQuestion question = new InterviewQuestion(title, difficulty, tags, link);
        question.setSource("mianshiya.com");
        
        return question;
    }
    
    /**
     * 爬取多页数据
     * @param startPage 开始页码
     * @param endPage 结束页码
     * @return 所有面试题目列表
     */
    public List<InterviewQuestion> crawlMultiplePages(int startPage, int endPage) {
        List<InterviewQuestion> allQuestions = new ArrayList<>();
        int totalPages = endPage - startPage + 1;
        
        log.info("开始爬取第{}页到第{}页，共{}页", startPage, endPage, totalPages);
        
        for (int page = startPage; page <= endPage; page++) {
            List<InterviewQuestion> pageQuestions = crawlPage(page);
            allQuestions.addAll(pageQuestions);
            
            // 每10页打印一次进度
            if (page % 10 == 0 || page == endPage) {
                log.info("进度: {}/{} 页完成，已获取{}条数据", page - startPage + 1, totalPages, allQuestions.size());
            }
            
            // 添加延迟，避免请求过于频繁
            if (page < endPage) {
                try {
                    TimeUnit.MILLISECONDS.sleep(DELAY_MS);
                } catch (InterruptedException e) {
                    log.warn("延迟被中断: {}", e.getMessage());
                    Thread.currentThread().interrupt();
                    break;
                }
            }
        }
        
        log.info("爬取完成，总共获取{}条数据", allQuestions.size());
        return allQuestions;
    }
    
    /**
     * 将数据保存到Excel文件
     * @param questions 面试题目列表
     * @param fileName 文件名
     */
    public void saveToExcel(List<InterviewQuestion> questions, String fileName) {
        try {
            String filePath = fileName + "_" + System.currentTimeMillis() + ".xlsx";
            EasyExcel.write(filePath, InterviewQuestion.class)
                    .sheet("面试题目")
                    .doWrite(questions);
            
            log.info("数据已保存到Excel文件: {}, 共{}条记录", filePath, questions.size());
        } catch (Exception e) {
            log.error("保存Excel文件失败: {}", e.getMessage());
        }
    }
    
    /**
     * 主方法，可以直接运行
     */
    public static void main(String[] args) {
        InterviewQuestionCrawler crawler = new InterviewQuestionCrawler();
        
        // 爬取全部500页数据
        log.info("开始爬取面试题目数据，预计爬取500页...");
        List<InterviewQuestion> questions = crawler.crawlMultiplePages(1, 20);
        
        if (!questions.isEmpty()) {
            // 保存到Excel
            crawler.saveToExcel(questions, "面试题目_全部数据");
            
            // 打印前几条数据作为示例
            log.info("爬取结果示例（前5条）:");
            for (int i = 0; i < Math.min(5, questions.size()); i++) {
                InterviewQuestion q = questions.get(i);
                log.info("第{}条数据:", i + 1);
                log.info("  题目: {}", q.getTitle());
                log.info("  难度: {}", q.getDifficulty());
                log.info("  标签: {}", q.getTags());
                log.info("  链接: {}", q.getLink());
                log.info("  ---");
            }
            
            log.info("总共爬取到{}条面试题目数据", questions.size());
        } else {
            log.warn("未获取到任何数据");
        }
        
        log.info("爬取任务完成");
    }
}
